package com.guiji.quartz.domain;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.CharUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.http.HtmlUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.json.JSONUtil;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.guiji.common.utils.spring.SpringUtils;
import com.guiji.quartz.config.ProxyConfig;
import com.guiji.quartz.formatter.DataTemplateFormatter;
import com.guiji.quartz.service.ISpiderNewsService;
import com.guiji.quartz.service.RedisService;
import com.guiji.quartz.task.DataSaveUtil;
import com.guiji.quartz.util.DateFormatUtil;
import com.guiji.quartz.util.DelPicUtil;
import lombok.Data;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.model.AfterExtractor;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.Formatter;
import us.codecraft.webmagic.model.annotation.TargetUrl;

import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: gaoX
 * @create: 2021-08-25 15:44
 **/
public class ChinaNews {
	private static final Logger log = LoggerFactory.getLogger(ChinaNews.class);

	public void mainMethod() throws Exception {
		HttpResponse response = null;
		try {
			response = HttpRequest.get("http://channel.chinanews.com/cns/cjs/sh.shtml")
				.header("Accept", "*/*")
				.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36")
				.header("X-Requested-With", "XMLHttpRequest")
				.header("Accept-Encoding", "gzip, deflate, br")
				.header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7")
				.setConnectionTimeout(15000)
				.execute();
		} catch (Exception e) {
			log.error("访问失败1" + e);
			return;
		}
		String resp = response.body();
		Set<String> resUrl = getUrl(resp);
		getContent(resUrl);

	}

	private static void getContent(Set<String> listContent) {
		for (String url : listContent) {
			log.info("getContent执行到-{},listContent长度-{}", url, listContent.size());
			String doc;
			JXDocument jxDocument;
			try {
				HttpResponse response = HttpRequest.get(url).setConnectionTimeout(15000).execute();
				doc = response.body();
				jxDocument = new JXDocument(doc);
			} catch (Exception e) {
				log.info("代理失败2" + e);
				continue;
			}
			String title = null;
			try {
				title = (String) jxDocument.sel("//h1/text()").get(0);
			} catch (Exception e) {
				log.error("标题错误-------------" + url + "-------------------标题错误");
				continue;
			}
			Document document = null;
			StringBuffer content = new StringBuffer();

			try {
				jxDocument.sel("//div[@class='left_zw']//p/text()").forEach(s -> {
					content.append(s+"$$$");
				});
			} catch (Exception e) {
				continue;
			}

			List<Object> newImgs = new ArrayList<>();
			try {
				jxDocument.sel("//div[@class='left_zw']//img/@src").forEach(img -> {
					newImgs.add(img);
				});
			} catch (Exception e) {
				log.error("获取图片失败");
			}

			Date date = null;
			try {
				DataTemplateFormatter dataTemplateFormatter = new DataTemplateFormatter();
				date = dataTemplateFormatter.format(jxDocument.sel("//div[@class='left_zw']//img/@src").get(0).toString());
			} catch (Exception e) {
				log.error("获取时间失败");
				date = new Date();
			}

			if (content.length() > 50000 || content.toString().replaceAll("\\$|[\\s\\p{Zs}]","").length() < 20 || newImgs.size() > 20) {
				continue;
			}
			try {
				int i = RandomUtil.randomInt(1, 5);
				DataSaveUtil.saveData(title, content.toString(), newImgs, url, date, "http://www.chinanews.com/society/", "中新网社会-"+i);
				log.info(Thread.currentThread().getName() + "-----------------中新网社会----------------");
			} catch (Exception e) {
				continue;
			}

		}
	}

	private static Set<String> getUrl(String content) {
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String s = sdf.format(new Date());
		String[] ss = sdf.format(new Date()).split("-");
		String year = ss[0];
		String mon = ss[1];
		String day = ss[2];

		Set<String> res = new HashSet<>();
		String pattern = "http://www.chinanews.com.cn/sh/"+year+"/"+mon+"-"+day+"/[0-9]{7}.shtml";
		Pattern p = Pattern.compile(pattern);
		Matcher m = p.matcher(content);
		while (m.find()) {
			res.add(m.group());
		}
		return res;
	}


	/*
	 * 过滤特殊字符串
	 * */
	public static String filterSpecialCharacter(String content) {
		// 1.还原被转义的HTML特殊字符
		content = HtmlUtil.unescape(content);
		// 2.清除所有HTML标签，但是保留标签内的内容
		content = HtmlUtil.cleanHtmlTag(content);

		// 3.处理一些特殊字符
		char[] blackCharList = new char[]{CharUtil.CR, '>', '<', 65279, 12288};
		content = StrUtil.removeAll(content,blackCharList);

		// 4.处理一些特殊字符串
		String[] blackStrList = new String[]{"</SPAN", "<SPAN", "<br", "</p", "<p", "<img", "<a", "/N"};
		for (String blackStr : blackStrList) {
			content = StrUtil.replaceIgnoreCase(content, blackStr, new String());
		}
		return content;
	}
}
